﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Collections;
using System.Net;
using System.IO;
using System.Text.RegularExpressions;

namespace Crawler
{
    public class Record
    {
        public int index;
        public string http;
        public Record(int index, string http)
        {
            this.http = http;
            this.index = index;
        }
    }
    public class SimpleCrawler
    {
        public System.Action refresh;
        public Hashtable urls = new Hashtable();
        public int count = 0;
        public ArrayList Rrecords = new ArrayList();
        public ArrayList Wrecords = new ArrayList();

        public SimpleCrawler(System.Action action) {
            refresh = action;
        }

        public void Crawle()
        {
            Regex isHtml = new Regex(@"<!DOCTYPE html>");
            while (true)
            {
                string current = null;
                foreach (string url in urls.Keys)
                {
                    if ((bool)urls[url]) continue;
                    current = url;
                }

                if (current == null || count > 100) break;

                string html = DownLoad(current);

                urls[current] = true;

                count++;

                if (isHtml.IsMatch(html))
                {
                    Parse(html);
                }
            }
        }

        public string DownLoad(string url)
        {
            try
            {
                WebClient webClient = new WebClient();
                webClient.Encoding = Encoding.UTF8;
                string html = webClient.DownloadString(url);

                string fileName = count.ToString();
                File.WriteAllText(fileName, html, Encoding.UTF8);

                return html;
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
                return "";
            }
        }

        public void Parse(string html)
        {
            string strRef = "(?<=(href|HREF)=\")http\\S+(?=\")";

            Regex isHttp = new Regex(@"(?<=http)\S+");

            MatchCollection matches = new Regex(strRef).Matches(html);

            foreach (Match match in matches)
            {
                strRef = match.Value.Substring(match.Value.IndexOf('=') + 1).Trim('"', '\"', '#', '>');

                if (strRef.Length == 0) continue;

                if (urls[strRef] == null && isHttp.IsMatch(strRef)) {
                    urls[strRef] = false;
                    Rrecords.Add(new Record(Rrecords.Count + 1, strRef));
                    refresh.Invoke();
                }
                else
                {
                    Wrecords.Add(new Record(Wrecords.Count + 1, strRef));
                    refresh.Invoke();
                }
            }
        }
    }
}
